# -*- coding: utf-8 -*-
# 爬取教务处
# 还可以爬取研究生院http://pg.njupt.edu.cn/s/26/t/775/p/1/c/8382/list.htm
# 图书馆爬取http://lib.njupt.edu.cn/s/36/t/472/p/1/c/4934/list.htm
__author__ = 'insect'
import requests
from bs4 import BeautifulSoup

import pymongo
import config

mongo_connection = pymongo.Connection(config.MongoHOST, config.MongoPORT)
db=mongo_connection.example
jwc_test=db.secondtest

JwcBaseUrl = "http://jwc.njupt.edu.cn"


# def get_page(url):
#     response = requests.get(url)
#     response.encoding = "utf-8"
#     return response.text

# 学生表格http://jwc.njupt.edu.cn/s/24/t/923/p/1/c/5495/d/5517/list.htm
# 教师表格http://jwc.njupt.edu.cn/s/24/t/923/p/1/c/5495/d/5549/list.htm
class DownloadJwcForm():
    def __init__(self):
        self.base_url = "http://jwc.njupt.edu.cn/s/24/t/923/p/1/c/5495/d/"
        self.base_url_num_dic = {
            "Student": [5517, 5533],
            "Teacher": [5549, 5563],
            "TeachingManagement": [5577, 5578, 5579, 5580, 5581, 5582, 5583, 5584, 5585,
                                   5586, 5587, 5588, 5589, 5598, 5599, 5560, 5561, 5562, 5563, 5604]
        }
        self.result_list = []

    def get_page(self, url):
        response = requests.get(url)
        response.encoding = "utf-8"
        html = BeautifulSoup(response.text)
        return html

    def make_url(self, number):
        return self.base_url + str(number) + "/list.htm"

    def parse_real_html(self, html, form_url):
        soup = html
        # print soup.title.string
        title = soup.title.string
        if title[-1:] == '\n':
            title = title[:-1]
        if title[-2:] == '\r\n':
            title = title[:-2]
        # print title
        if soup.find("a", id="file1") is None:
            the_dic = {
                "title": title,
                "url": form_url,
                "identify": "jwc form"
            }
            self.result_list.append(the_dic)
            # print form_url
        else:
            table_list = soup.find("td", class_="content")
            a_list = table_list.find_all("a")
            for item in a_list:
                the_dic = {
                    "title": item.string,
                    "url": (JwcBaseUrl + item['href']),
                    "identify": "jwc form",
                }
                self.result_list.append(the_dic)
            # the_dic = {
            #     "title": title,
            #     "url": (JwcBaseUrl + soup.find("a", id="file1")['href']),
            #     "identify": "jwc form"
            # }
            # self.result_list.append(the_dic)
            # print soup.find("a", id="file1")['href']

    def parse_list_html(self, html):
        soup = html
        the_list = soup.find_all("table", class_="columnStyle")
        if len(the_list) == 0:
            pass
        for item in the_list:
            if str(item.a['href'])[-4:] == '.doc':
                title = item.font.string
                if title[-1:] == '\n':
                    title = title[:-1]
                if title[-2:] == '\r\n':
                    title = title[:-2]
                the_dic = {
                    "title": title,
                    "url": (JwcBaseUrl + item.a['href']),
                    "identify": "jwc form"
                }
                self.result_list.append(the_dic)
                # print item.font.string
                # print item.a['href']
                continue
            else:
                if str(item.a['href'])[:4] == 'http':
                    form_url = str(item.a['href'])
                else:
                    form_url = JwcBaseUrl + str(item.a['href'])
                self.parse_real_html(self.get_page(form_url), form_url)

    def get(self):
        for key in self.base_url_num_dic:
            if len(self.base_url_num_dic[key]) == 2:
                for num in range(self.base_url_num_dic[key][0], self.base_url_num_dic[key][1]):
                    self.parse_list_html(self.get_page(self.make_url(num)))
            else:
                for num in self.base_url_num_dic[key]:
                    self.parse_list_html(self.get_page(self.make_url(num)))


    def get_result(self):
        self.get()
        return self.result_list



if __name__ == '__main__':
    test = DownloadJwcForm()
    result = test.get_result()
    for item in result:
        jwc_test.insert(item)
    # search_result = jwc_test.find({"title": '团队优秀毕业设计(论文)推荐表.doc'})
    # print search_result
    # new_search_result = []
    # repetition_list = []
    # for item in search_result:
    #     if item['url'] not in repetition_list:
    #         new_search_result.append(item)
    #         repetition_list.append(item['url'])
    #     else:
    #         continue
    # for item in new_search_result:
    #     print item['title']
    #     print item['url']
    # parse_html(get_page("http://jwc.njupt.edu.cn/s/24/t/923/7b/85/info97157.htm"))